The protected file system is first initialized by function pal_linux_main (in db_main.c).

PF initialization

The init function is init_protected_files in enclave_pf.c.

int init_protected_files(void) {
    int ret;
    pf_debug_f debug_callback = NULL;

#ifdef DEBUG
    debug_callback = cb_debug;
#endif

    pf_set_callbacks(cb_read, cb_write, cb_truncate, cb_aes_gcm_encrypt, cb_aes_gcm_decrypt,
                     cb_random, debug_callback);

    /* if wrap key is not hard-coded in the manifest, assume that it was received from parent or
     * it will be provisioned after local/remote attestation; otherwise read it from manifest */
    char* protected_files_key_str = NULL;
    ret = toml_string_in(g_pal_state.manifest_root, "sgx.protected_files_key",
                         &protected_files_key_str);
    if (ret < 0) {
        log_error("Cannot parse \'sgx.protected_files_key\' "
                  "(the value must be put in double quotes!)\n");
        return -PAL_ERROR_INVAL;
    }

    if (protected_files_key_str) {
        if (strlen(protected_files_key_str) != PF_KEY_SIZE * 2) {
            log_error("Malformed \'sgx.protected_files_key\' value in the manifest\n");
            free(protected_files_key_str);
            return -PAL_ERROR_INVAL;
        }

        memset(g_pf_wrap_key, 0, sizeof(g_pf_wrap_key));
        for (size_t i = 0; i < strlen(protected_files_key_str); i++) {
            int8_t val = hex2dec(protected_files_key_str[i]);
            if (val < 0) {
                log_error("Malformed \'sgx.protected_files_key\' value in the manifest\n");
                free(protected_files_key_str);
                return -PAL_ERROR_INVAL;
            }
            g_pf_wrap_key[i / 2] = g_pf_wrap_key[i / 2] * 16 + (uint8_t)val;
        }

        free(protected_files_key_str);
        g_pf_wrap_key_set = true;
    }

    if (register_protected_files() < 0) {
        log_error("Malformed protected files found in manifest\n");
    }

    return 0;
}

Modify this function if RA/LA to pass the key to PFs

1. Set callback functions which will be invoked for encryption and decryption (when are they called?)
2. Read PF key from manifest
3. Register protected files

Open in Graphene SGX

It will check if the file to open is a PF first. See code in db_files.c:

static int file_open(PAL_HANDLE* handle, const char* type, const char* uri, int access, int share,
                     int create, int options) {
// MISSING CODE ...

        pf = load_protected_file(path, (int*)&hdl->file.fd, st.st_size, pf_mode, pf_create, pf);
        if (pf) {
            pf->refcount++;
            if (pf_mode & PF_FILE_MODE_WRITE) {
                pf->writable_fd = fd;
            }
        } else {

// MISSING CODE ...

See also Graphene-SGX PF Implementation for more details.

/* Host callbacks */
static pf_read_f     g_cb_read     = NULL;
static pf_write_f    g_cb_write    = NULL;
static pf_truncate_f g_cb_truncate = NULL;
static pf_debug_f    g_cb_debug    = NULL;

static pf_aes_gcm_encrypt_f g_cb_aes_gcm_encrypt = NULL;
static pf_aes_gcm_decrypt_f g_cb_aes_gcm_decrypt = NULL;
static pf_random_f          g_cb_random          = NULL;

read

g_cb_read is used in ipf_read_node:

static bool ipf_read_node(pf_context_t* pf, pf_handle_t handle, uint64_t node_number, void* buffer,
                          uint32_t node_size) {
    uint64_t offset = node_number * node_size;

    pf_status_t status = g_cb_read(handle, buffer, offset, node_size);
    if (PF_FAILURE(status)) {
        pf->last_error = status;
        return false;
    }

    return true;
}

This callback function is used in PF:

/* Callbacks for protected files handling */
static pf_status_t cb_read(pf_handle_t handle, void* buffer, uint64_t offset, size_t size) {
    int fd = *(int*)handle;
    size_t buffer_offset = 0;
    size_t to_read = size;
    while (to_read > 0) {
        ssize_t read = ocall_pread(fd, buffer + buffer_offset, to_read, offset + buffer_offset);
        if (read == -EINTR)
            continue;

        if (read < 0) {
            log_error("cb_read(%d, %p, %lu, %lu): read failed: %ld\n", fd, buffer, offset,
                      size, read);
            return PF_STATUS_CALLBACK_FAILED;
        }

        /* EOF is an error condition, we want to read exactly `size` bytes */
        if (read == 0) {
            log_error("cb_read(%d, %p, %lu, %lu): EOF\n", fd, buffer, offset, size);
            return PF_STATUS_CALLBACK_FAILED;
        }

        to_read -= read;
        buffer_offset += read;
    }
    return PF_STATUS_SUCCESS;
}

write

g_cb_write is used in ipf_write_file.

static bool ipf_write_file(pf_context_t* pf, pf_handle_t handle, uint64_t offset, void* buffer,
                           uint32_t size) {
    pf_status_t status = g_cb_write(handle, buffer, offset, size);
    if (PF_FAILURE(status)) {
        pf->last_error = status;
        return false;
    }

    return true;
}

This callback function is used in PF:

static pf_status_t cb_write(pf_handle_t handle, const void* buffer, uint64_t offset, size_t size) {
    int fd = *(int*)handle;
    size_t buffer_offset = 0;
    size_t to_write = size;
    while (to_write > 0) {
        ssize_t written = ocall_pwrite(fd, buffer + buffer_offset, to_write,
                                       offset + buffer_offset);
        if (written == -EINTR)
            continue;

        if (written < 0) {
            log_error("cb_write(%d, %p, %lu, %lu): write failed: %ld\n", fd, buffer, offset,
                      size, written);
            return PF_STATUS_CALLBACK_FAILED;
        }

        /* EOF is an error condition, we want to write exactly `size` bytes */
        if (written == 0) {
            log_error("cb_write(%d, %p, %lu, %lu): EOF\n", fd, buffer, offset, size);
            return PF_STATUS_CALLBACK_FAILED;
        }

        to_write -= written;
        buffer_offset += written;
    }
    return PF_STATUS_SUCCESS;
}

truncate

Not used in the implementation, it's less interesting.

encrypt

g_cb_aes_gcm_encrypt is used in several places, including ipf_import_metadata_key, ipf_update_all_data_and_mht_nodes, ipf_update_all_data_and_mht_nodes, ipf_update_metadata_node, all in file protected_files.c. This function is mainly called when data needs updates or init.

The callback is merely calling AES GCM function to encrypt.

static pf_status_t cb_aes_gcm_encrypt(const pf_key_t* key, const pf_iv_t* iv, const void* aad,
                                      size_t aad_size, const void* input, size_t input_size,
                                      void* output, pf_mac_t* mac) {
    int ret = lib_AESGCMEncrypt((const uint8_t*)key, sizeof(*key), (const uint8_t*)iv, input,
                                input_size, aad, aad_size, output, (uint8_t*)mac, sizeof(*mac));
    if (ret != 0) {
        log_error("lib_AESGCMEncrypt failed: %d\n", ret);
        return PF_STATUS_CALLBACK_FAILED;
    }
    return PF_STATUS_SUCCESS;
}

decrypt

g_cb_aes_gcm_decrypt is used in ipf_init_existing_file, ipf_read_data_node, ipf_read_mht_node.

Likewise, its callback is:

static pf_status_t cb_aes_gcm_decrypt(const pf_key_t* key, const pf_iv_t* iv, const void* aad,
                                      size_t aad_size, const void* input, size_t input_size,
                                      void* output, const pf_mac_t* mac) {
    int ret = lib_AESGCMDecrypt((const uint8_t*)key, sizeof(*key), (const uint8_t*)iv, input,
                                input_size, aad, aad_size, output, (const uint8_t*)mac,
                                sizeof(*mac));
    if (ret != 0) {
        log_error("lib_AESGCMDecrypt failed: %d\n", ret);
        return PF_STATUS_CALLBACK_FAILED;
    }
    return PF_STATUS_SUCCESS;
}

random

g_cb_random is used in ipf_import_metadata_key, ipf_generate_random_key.

The callback calls _DkRandomBitsRead, which reads in random bits from a random device.

static pf_status_t cb_random(uint8_t* buffer, size_t size) {
    int ret = _DkRandomBitsRead(buffer, size);
    if (ret < 0) {
        log_error("_DkRandomBitsRead failed: %d\n", ret);
        return PF_STATUS_CALLBACK_FAILED;
    }
    return PF_STATUS_SUCCESS;
}

pf_contex

struct pf_context {
    metadata_node_t file_metadata; // actual data from disk's meta data node
    pf_status_t last_error;
    metadata_encrypted_t encrypted_part_plain; // encrypted part of metadata node, decrypted
    file_node_t root_mht; // the root of the mht is always needed (for files bigger than 3KB)
    pf_handle_t file;
    pf_file_mode_t mode;
    uint64_t offset; // current file position (user's view)
    bool end_of_file;
    uint64_t real_file_size;
    bool need_writing;
    pf_status_t file_status;
    pf_key_t user_kdk_key;
    pf_key_t cur_key;
    lruc_context_t* cache;
#ifdef DEBUG
    char* debug_buffer; // buffer for debug output
#endif
};

See also: SGX_PF_Class

How nodes organized

In the PF implementation, there is a function called get_node_numbers, and the comments implies the node structure of a PF:

• What's child MHT node? why 32?
• Three types of nodes: metadata, data, and mht
• metadata: TODO
• data: just to store encrypted data
• mht: each mht node can contain key and gmac of 96 data nodes and 32 child mht nodes. Every pf has at least one root mht node, and a mht (child) node is added to this pf after every 96 consecutive data nodes.

// this is a very 'specific' function, tied to the architecture of the file layout,
// returning the node numbers according to the data offset in the file
static void get_node_numbers(uint64_t offset, uint64_t* mht_node_number, uint64_t* data_node_number,
                             uint64_t* physical_mht_node_number,
                             uint64_t* physical_data_node_number) {
    // physical nodes (file layout):
    // node 0 - meta data node
    // node 1 - mht
    // nodes 2-97 - data (ATTACHED_DATA_NODES_COUNT == 96)
    // node 98 - mht
    // node 99-195 - data
    // etc.
    uint64_t _physical_mht_node_number;
    uint64_t _physical_data_node_number;

    // "logical" nodes: sequential index of the corresponding mht/data node in all mht/data nodes
    uint64_t _mht_node_number;
    uint64_t _data_node_number;

    assert(offset >= MD_USER_DATA_SIZE);

    _data_node_number = (offset - MD_USER_DATA_SIZE) / PF_NODE_SIZE;
    _mht_node_number = _data_node_number / ATTACHED_DATA_NODES_COUNT;
    _physical_data_node_number = _data_node_number
                                 + 1 // meta data node
                                 + 1 // mht root
                                 + _mht_node_number; // number of mht nodes in the middle
                                 // (the root mht mht_node_number is 0)
    _physical_mht_node_number = _physical_data_node_number
                                - _data_node_number % ATTACHED_DATA_NODES_COUNT // now we are at
                                // the first data node attached to this mht node
                                - 1; // and now at the mht node itself!

    if (mht_node_number != NULL)
        *mht_node_number = _mht_node_number;
    if (data_node_number != NULL)
        *data_node_number = _data_node_number;
    if (physical_mht_node_number != NULL)
        *physical_mht_node_number = _physical_mht_node_number;
    if (physical_data_node_number != NULL)
        *physical_data_node_number = _physical_data_node_number;
}

file_node_t

• This macro also defines a linked list of file_node_t
• Two types of decrypted data in this node

DEFINE_LIST(_file_node);
typedef struct _file_node {
    LIST_TYPE(_file_node) list;
    uint8_t type;
    uint64_t node_number;
    struct _file_node* parent;
    bool need_writing;
    bool new_node;
    struct {
        uint64_t physical_node_number;
        encrypted_node_t encrypted; // the actual data from the disk
    };
    union { // decrypted data
        mht_node_t mht;
        data_node_t data;
    } decrypted;
} file_node_t;
DEFINE_LISTP(_file_node);

typedef struct _mht_node {
    gcm_crypto_data_t data_nodes_crypto[ATTACHED_DATA_NODES_COUNT];
    gcm_crypto_data_t mht_nodes_crypto[CHILD_MHT_NODES_COUNT];
} mht_node_t;

typedef struct _data_node {
    uint8_t data[PF_NODE_SIZE];
} data_node_t;

typedef struct _encrypted_node {
    uint8_t cipher[PF_NODE_SIZE];
} encrypted_node_t;

See also: SDK_PF_DataNode

metadata_node_t

• It seems like in graphene pf, the metadata node will be padded rather than filled in with user data.
• Similar to SDK: #define PF_NODE_SIZE 4096U and #define MD_USER_DATA_SIZE (PF_NODE_SIZE * 3 / 4)

typedef struct _metadata_node {
    metadata_plain_t          plain_part;
    metadata_encrypted_blob_t encrypted_part;
    metadata_padding_t        padding;
} metadata_node_t;

typedef struct _metadata_plain {
    uint64_t   file_id;
    uint8_t    major_version;
    uint8_t    minor_version;
    pf_keyid_t metadata_key_id;
    pf_mac_t   metadata_gmac; /* GCM mac */
} metadata_plain_t;

typedef struct _metadata_encrypted {
    char     path[PATH_MAX_SIZE];
    uint64_t size;
    pf_key_t mht_key;
    pf_mac_t mht_gmac;
    uint8_t  data[MD_USER_DATA_SIZE];
} metadata_encrypted_t;

typedef uint8_t metadata_encrypted_blob_t[sizeof(metadata_encrypted_t)];

typedef uint8_t metadata_padding_t[METADATA_NODE_SIZE -
                                   (sizeof(metadata_plain_t) + sizeof(metadata_encrypted_blob_t))];

See also: SDK_PF_metadata

High-level Overview

1. Protected file operations trapped into the libOS
2. LibOS calls the corresponding user-level functions (pf_*)
3. The user-level functions invokes it's internal Intel PF functions (ipf_*)
4. Callback the actual functions which does the operation (cb_*)

• PF is stateful and it's maintained by pf_context
• Integrity guaranteed by MHT

Gadgets

• The implementation of PF can be found here.
• Just like the Intel PF, there are a set of user interfaces and another set of functions as internal implementation.
• ipf stands for Intel Protected File. They are internal implementation of PF, just like which in the SGX SDK.

User Interfaces

Graphene-SGX PF User Interfaces

Callbacks

First of all, several callback function pointers are set beforehand and will be used later in read & write, encryption and decryption.

All the call back function pointers are set to callbacks from enclave_pf.c.

• Graphene-SGX PF Callbacks

Internal data structures

They are very similar to which in SGX SDK.

• Graphene-SGX PF Data Structure

Internal implementation

Functions starting with name ipf_.

Graphene-SGX PF Internal

These functions performs essential operations on protected files. The implementations seem like transporting the official Intel SGX SDK cpp code to c at graphene side.

• LRU cache is used for data/mht nodes iteration

ipf_open

1. Create and initialize(ipf_init_fields) a pf(pf_context_t)
2. Check validity of key, path, handle(fd), size
3. Check if it needs to create a new file or the file exits
4. Return pf

static pf_context_t* ipf_open(const char* path, pf_file_mode_t mode, bool create, pf_handle_t file,
                              uint64_t real_size, const pf_key_t* kdk_key, pf_status_t* status) {
    *status = PF_STATUS_NO_MEMORY;
    pf_context_t* pf = calloc(1, sizeof(*pf));

    if (!pf)
        goto out;

    if (!ipf_init_fields(pf))
        goto out;

    DEBUG_PF("handle: %d, path: '%s', real size: %lu, mode: 0x%x\n", *(int*)file, path, real_size,
             mode);

    if (kdk_key == NULL) {
        DEBUG_PF("no key specified\n");
        pf->last_error = PF_STATUS_INVALID_PARAMETER;
        goto out;
    }

    if (path && strlen(path) > PATH_MAX_SIZE - 1) {
        pf->last_error = PF_STATUS_PATH_TOO_LONG;
        goto out;
    }

    // for new file, this value will later be saved in the meta data plain part (init_new_file)
    // for existing file, we will later compare this value with the value from the file
    // (init_existing_file)
    COPY_ARRAY(pf->user_kdk_key, *kdk_key);

    // omeg: we require a canonical full path to file, so no stripping path to filename only
    // omeg: Intel's implementation opens the file, we get the fd and size from the Graphene handler

    if (!file) {
        DEBUG_PF("invalid handle\n");
        pf->last_error = PF_STATUS_INVALID_PARAMETER;
        goto out;
    }

    if (real_size % PF_NODE_SIZE != 0) {
        pf->last_error = PF_STATUS_INVALID_HEADER;
        goto out;
    }

    pf->file = file;
    pf->real_file_size = real_size;
    pf->mode = mode;

    if (!create) {
        // existing file
        if (!ipf_init_existing_file(pf, path))
            goto out;

    } else {
        // new file
        if (!ipf_init_new_file(pf, path))
            goto out;
    }

    pf->last_error = pf->file_status = PF_STATUS_SUCCESS;
    DEBUG_PF("OK (data size %lu)\n", pf->encrypted_part_plain.size);

out:
    if (pf && PF_FAILURE(pf->last_error)) {
        DEBUG_PF("failed: %d\n", pf->last_error);
        free(pf);
        pf = NULL;
    }

    if (pf)
        *status = pf->last_error;

    return pf;
}

ipf_read

1. Check if error occurs beforehand
2. Get data out from metadata block (1st blk) if needed
3. Continuously retrieve a node to read and read this node (ipf_get_data_node）

static size_t ipf_read(pf_context_t* pf, void* ptr, size_t size) {
    if (ptr == NULL) {
        pf->last_error = PF_STATUS_INVALID_PARAMETER;
        return 0;
    }

    if (PF_FAILURE(pf->file_status)) {
        pf->last_error = pf->file_status;
        return 0;
    }

    if (!(pf->mode & PF_FILE_MODE_READ)) {
        pf->last_error = PF_STATUS_INVALID_MODE;
        return 0;
    }

    size_t data_left_to_read = size;

    if (((uint64_t)data_left_to_read) > (uint64_t)(pf->encrypted_part_plain.size - pf->offset)) {
        // the request is bigger than what's left in the file
        data_left_to_read = (size_t)(pf->encrypted_part_plain.size - pf->offset);
    }

    // used at the end to return how much we actually read
    size_t data_attempted_to_read = data_left_to_read;

    unsigned char* out_buffer = (unsigned char*)ptr;

    // the first block of user data is read from the meta-data encrypted part
    if (pf->offset < MD_USER_DATA_SIZE) {
        // offset is smaller than MD_USER_DATA_SIZE
        size_t data_left_in_md = MD_USER_DATA_SIZE - (size_t)pf->offset;
        size_t size_to_read = MIN(data_left_to_read, data_left_in_md);

        memcpy(out_buffer, &pf->encrypted_part_plain.data[pf->offset], size_to_read);
        pf->offset += size_to_read;
        out_buffer += size_to_read;
        data_left_to_read -= size_to_read;
    }

    while (data_left_to_read > 0) {
        file_node_t* file_data_node = NULL;
        // return the data node of the current offset, will read it from disk if needed
        // (and also the mht node if needed)
        file_data_node = ipf_get_data_node(pf);
        if (file_data_node == NULL)
            break;

        size_t offset_in_node = (pf->offset - MD_USER_DATA_SIZE) % PF_NODE_SIZE;
        size_t data_left_in_node = PF_NODE_SIZE - offset_in_node;
        size_t size_to_read = MIN(data_left_to_read, data_left_in_node);

        memcpy(out_buffer, &file_data_node->decrypted.data.data[offset_in_node], size_to_read);
        pf->offset += size_to_read;
        out_buffer += size_to_read;
        data_left_to_read -= size_to_read;
    }

    if (data_left_to_read == 0 && data_attempted_to_read != size) {
        // user wanted to read more and we had to shrink the request
        assert(pf->offset == pf->encrypted_part_plain.size);
        pf->end_of_file = true;
    }

    return data_attempted_to_read - data_left_to_read;
}

static bool ipf_read_node(pf_context_t* pf, pf_handle_t handle, uint64_t node_number, void* buffer,
                          uint32_t node_size) {
    uint64_t offset = node_number * node_size;

    pf_status_t status = g_cb_read(handle, buffer, offset, node_size);
    if (PF_FAILURE(status)) {
        pf->last_error = status;
        return false;
    }

    return true;
}

ipf_write

1. Check if error occurred
2. Write to metadata block if there is empty space left
3. Get data node (ipf_get_data_node) and write data into the node iteratively. Besides, set need_writing to all mht nodes in this pf.

// write zeros if `ptr` is NULL
static size_t ipf_write(pf_context_t* pf, const void* ptr, size_t size) {
    if (size == 0) {
        pf->last_error = PF_STATUS_INVALID_PARAMETER;
        return 0;
    }

    size_t data_left_to_write = size;

    if (PF_FAILURE(pf->file_status)) {
        pf->last_error = pf->file_status;
        DEBUG_PF("bad file status %d\n", pf->last_error);
        return 0;
    }

    if (!(pf->mode & PF_FILE_MODE_WRITE)) {
        pf->last_error = PF_STATUS_INVALID_MODE;
        DEBUG_PF("File is read-only\n");
        return 0;
    }

    const unsigned char* data_to_write = (const unsigned char*)ptr;

    // the first block of user data is written in the meta-data encrypted part
    if (pf->offset < MD_USER_DATA_SIZE) {
        // offset is smaller than MD_USER_DATA_SIZE
        size_t empty_place_left_in_md = MD_USER_DATA_SIZE - (size_t)pf->offset;
        size_t size_to_write = MIN(data_left_to_write, empty_place_left_in_md);

        memcpy_or_zero_initialize(&pf->encrypted_part_plain.data[pf->offset], data_to_write,
                                  size_to_write);
        pf->offset += size_to_write;
        if (data_to_write)
            data_to_write += size_to_write;
        data_left_to_write -= size_to_write;

        if (pf->offset > pf->encrypted_part_plain.size)
            pf->encrypted_part_plain.size = pf->offset; // file grew, update the new file size

        pf->need_writing = true;
    }

    while (data_left_to_write > 0) {
        file_node_t* file_data_node = NULL;
        // return the data node of the current offset, will read it from disk or create new one
        // if needed (and also the mht node if needed)
        file_data_node = ipf_get_data_node(pf);
        if (file_data_node == NULL) {
            DEBUG_PF("failed to get data node\n");
            break;
        }

        size_t offset_in_node = (size_t)((pf->offset - MD_USER_DATA_SIZE) % PF_NODE_SIZE);
        size_t empty_place_left_in_node = PF_NODE_SIZE - offset_in_node;
        size_t size_to_write = MIN(data_left_to_write, empty_place_left_in_node);

        memcpy_or_zero_initialize(&file_data_node->decrypted.data.data[offset_in_node],
                                  data_to_write, size_to_write);
        pf->offset += size_to_write;
        if (data_to_write)
            data_to_write += size_to_write;
        data_left_to_write -= size_to_write;

        if (pf->offset > pf->encrypted_part_plain.size) {
            pf->encrypted_part_plain.size = pf->offset; // file grew, update the new file size
        }

        if (!file_data_node->need_writing) {
            file_data_node->need_writing = true;
            file_node_t* file_mht_node = file_data_node->parent;
            while (file_mht_node->node_number != 0) {
                // set all the mht parent nodes as 'need writing'
                file_mht_node->need_writing = true;
                file_mht_node = file_mht_node->parent;
            }
            pf->root_mht.need_writing = true;
            pf->need_writing = true;
        }
    }

    return size - data_left_to_write;
}

ipf_get_data_node

1. check the offset is at the edge of nodes and not in metanode
2. If offset==size, create a new node by ipf_append_data_node; otherwise read the next data node ipf_read_data_node.
3. "bump all the parents mht to reside before the data node in the cache"
4. Do other LRU cache related operations TODO

static file_node_t* ipf_get_data_node(pf_context_t* pf) {
    file_node_t* file_data_node = NULL;

    if (pf->offset < MD_USER_DATA_SIZE) {
        pf->last_error = PF_STATUS_UNKNOWN_ERROR;
        return NULL;
    }

    if ((pf->offset - MD_USER_DATA_SIZE) % PF_NODE_SIZE == 0
        && pf->offset == pf->encrypted_part_plain.size) {
        // new node
        file_data_node = ipf_append_data_node(pf);
    } else {
        // existing node
        file_data_node = ipf_read_data_node(pf);
    }

    // bump all the parents mht to reside before the data node in the cache
    if (file_data_node != NULL) {
        file_node_t* file_mht_node = file_data_node->parent;
        while (file_mht_node->node_number != 0) {
            // bump the mht node to the head of the lru
            lruc_get(pf->cache, file_mht_node->physical_node_number);
            file_mht_node = file_mht_node->parent;
        }
    }

    // even if we didn't get the required data_node, we might have read other nodes in the process
    while (lruc_size(pf->cache) > MAX_PAGES_IN_CACHE) {
        void* data = lruc_get_last(pf->cache);
        assert(data != NULL);
        // for production -
        if (data == NULL) {
            pf->last_error = PF_STATUS_UNKNOWN_ERROR;
            return NULL;
        }

        if (!((file_node_t*)data)->need_writing) {
            lruc_remove_last(pf->cache);

            // before deleting the memory, need to scrub the plain secrets
            file_node_t* file_node = (file_node_t*)data;
            erase_memory(&file_node->decrypted, sizeof(file_node->decrypted));
            free(file_node);
        } else {
            if (!ipf_internal_flush(pf)) {
                // error, can't flush cache, file status changed to error
                assert(pf->file_status != PF_STATUS_SUCCESS);
                if (pf->file_status == PF_STATUS_SUCCESS)
                    pf->file_status = PF_STATUS_FLUSH_ERROR; // for release set this anyway
                return NULL;                                 // even if we got the data_node!
            }
        }
    }

    return file_data_node;
}

ipf_read_data_node

This function first reads the ciphertext of a node by specifying a node number and a pf, then decrypt the ciphertext using the callback function to get the plaintext. Besides, add what have been read to LRU cache.

ipf_close

• called by pf_close
• invokes ipf_internal_flush if the status is not PF_STATUS_SUCCESS
• also do some cleaning jobs on the LRU cache

ipf_internal_flush

1. Check if writing is needed (for mht and other nodes)
2. Update data/mht/meta nodes
3. Write to disk

static bool ipf_internal_flush(pf_context_t* pf) {
    if (!pf->need_writing) {
        // no changes at all
        DEBUG_PF("no need to write\n");
        return true;
    }

    if (pf->encrypted_part_plain.size > MD_USER_DATA_SIZE && pf->root_mht.need_writing) {
        // otherwise it's just one write - the meta-data node
        if (!ipf_update_all_data_and_mht_nodes(pf)) {
            // this is something that shouldn't happen, can't fix this...
            pf->file_status = PF_STATUS_CRYPTO_ERROR;
            DEBUG_PF("failed to update data nodes\n");
            return false;
        }
    }

    if (!ipf_update_metadata_node(pf)) {
        // this is something that shouldn't happen, can't fix this...
        pf->file_status = PF_STATUS_CRYPTO_ERROR;
        DEBUG_PF("failed to update metadata nodes\n");
        return false;
    }

    if (!ipf_write_all_changes_to_disk(pf)) {
        pf->file_status = PF_STATUS_WRITE_TO_DISK_FAILED;

        DEBUG_PF("failed to write changes to disk\n");
        return false;
    }

    pf->need_writing = false;

    return true;
}

ipf_update_all_data_and_mht_nodes

This function is very complex. It iterates all nodes with need_writing flag set and do corresponding change/encryption/update

1. Iterate all data nodes needing writing: generate a key for the node, store the key in its parent mht node and encrypt data, then store the gmac in the mht node
2. Iterate all mht nodes needing writing: put these mht nodes into an array and sort them
3. Iterate this array from the last node to the first: like what have been done to the data nodes, mht nodes also need to be maintained in a similar way.

static bool ipf_update_all_data_and_mht_nodes(pf_context_t* pf) {
    bool ret = false;
    file_node_t** mht_array = NULL;
    file_node_t* file_mht_node;
    pf_status_t status;
    void* data = lruc_get_first(pf->cache);

    // 1. encrypt the changed data
    // 2. set the IV+GMAC in the parent MHT
    // [3. set the need_writing flag for all the parents]
    while (data != NULL) {
        if (((file_node_t*)data)->type == FILE_DATA_NODE_TYPE) {
            file_node_t* data_node = (file_node_t*)data;

            if (data_node->need_writing) {
                gcm_crypto_data_t* gcm_crypto_data =
                    &data_node->parent->decrypted.mht
                         .data_nodes_crypto[data_node->node_number % ATTACHED_DATA_NODES_COUNT];

                if (!ipf_generate_random_key(pf, &gcm_crypto_data->key))
                    goto out;

                // encrypt the data, this also saves the gmac of the operation in the mht crypto
                // node
                status = g_cb_aes_gcm_encrypt(&gcm_crypto_data->key, &g_empty_iv, NULL, 0,  // aad
                                              data_node->decrypted.data.data, PF_NODE_SIZE,
                                              data_node->encrypted.cipher, &gcm_crypto_data->gmac);
                if (PF_FAILURE(status)) {
                    pf->last_error = status;
                    goto out;
                }

                file_mht_node = data_node->parent;
#ifdef DEBUG
                // this loop should do nothing, add it here just to be safe
                while (file_mht_node->node_number != 0) {
                    assert(file_mht_node->need_writing == true);
                    file_mht_node = file_mht_node->parent;
                }
#endif
            }
        }
        data = lruc_get_next(pf->cache);
    }

    size_t dirty_count = 0;

    // count dirty mht nodes
    data = lruc_get_first(pf->cache);
    while (data != NULL) {
        if (((file_node_t*)data)->type == FILE_MHT_NODE_TYPE) {
            if (((file_node_t*)data)->need_writing)
                dirty_count++;
        }
        data = lruc_get_next(pf->cache);
    }

    // add all the mht nodes that needs writing to a list
    mht_array = malloc(dirty_count * sizeof(*mht_array));
    if (!mht_array) {
        pf->last_error = PF_STATUS_NO_MEMORY;
        goto out;
    }

    data = lruc_get_first(pf->cache);
    uint64_t dirty_idx = 0;
    while (data != NULL) {
        if (((file_node_t*)data)->type == FILE_MHT_NODE_TYPE) {
            file_mht_node = (file_node_t*)data;

            if (file_mht_node->need_writing)
                mht_array[dirty_idx++] = file_mht_node;
        }

        data = lruc_get_next(pf->cache);
    }

    if (dirty_count > 0)
        sort_nodes(mht_array, 0, dirty_count - 1);

    // update the gmacs in the parents from last node to first (bottom layers first)
    for (dirty_idx = dirty_count; dirty_idx > 0; dirty_idx--) {
        file_mht_node = mht_array[dirty_idx - 1];

        gcm_crypto_data_t* gcm_crypto_data =
            &file_mht_node->parent->decrypted.mht
                 .mht_nodes_crypto[(file_mht_node->node_number - 1) % CHILD_MHT_NODES_COUNT];

        if (!ipf_generate_random_key(pf, &gcm_crypto_data->key)) {
            goto out;
        }

        status = g_cb_aes_gcm_encrypt(&gcm_crypto_data->key, &g_empty_iv, NULL, 0,
                                      &file_mht_node->decrypted.mht, PF_NODE_SIZE,
                                      &file_mht_node->encrypted.cipher, &gcm_crypto_data->gmac);
        if (PF_FAILURE(status)) {
            pf->last_error = status;
            goto out;
        }
    }

    // update mht root gmac in the meta data node
    if (!ipf_generate_random_key(pf, &pf->encrypted_part_plain.mht_key))
        goto out;

    status = g_cb_aes_gcm_encrypt(&pf->encrypted_part_plain.mht_key, &g_empty_iv,
                                  NULL, 0,
                                  &pf->root_mht.decrypted.mht, PF_NODE_SIZE,
                                  &pf->root_mht.encrypted.cipher,
                                  &pf->encrypted_part_plain.mht_gmac);
    if (PF_FAILURE(status)) {
        pf->last_error = status;
        goto out;
    }

    ret = true;

out:
    free(mht_array);
    return ret;
}
''''

!! `ipf_update_metadata_node`

This function first generate a random key (derived from user's KDK) and encrypt the metadata node using this key. 

!! `ipf_write_all_changes_to_disk`

Write all nodes in `pf` back to the disk. It will first write data and child mht nodes, then root mht node, and last, write the metadata node.

All these functions will first check if the PF subsystem is correctly initialized (callback functions set).

pf_open

First we need to know how a PF is opened.

pf_status_t pf_open(pf_handle_t handle, const char* path, uint64_t underlying_size,
                    pf_file_mode_t mode, bool create, const pf_key_t* key, pf_context_t** context) {
    if (!g_initialized)
        return PF_STATUS_UNINITIALIZED;

    pf_status_t status;
    *context = ipf_open(path, mode, create, handle, underlying_size, key, &status);
    return status;
}

Function call trace:

• file_open ->
• load_protected_file ->
• open_protected_file ->
• pf_open ->
• ipf_open

pf_close

Just like pf_open, this function will be invoked in unload_protected_file or when error occurs and file needs to be closed.

pf_status_t pf_close(pf_context_t* pf) {
    if (!g_initialized)
        return PF_STATUS_UNINITIALIZED;

    if (ipf_close(pf))
        return PF_STATUS_SUCCESS;
    return pf->last_error;
}

pf_get_size

This function is invoked in function related to file mapping, like pf_file_map in db_files.c. Not of great interest.

pf_set_size

To truncate a file, also not very interesting.

pf_read

pf_read and pf_wirte have similar logic: seek to an appropriate position in the file and then read or wirte.

pf_status_t pf_read(pf_context_t* pf, uint64_t offset, size_t size, void* output,
                    size_t* bytes_read) {
    if (!g_initialized)
        return PF_STATUS_UNINITIALIZED;

    if (!size) {
        *bytes_read = 0;
        return PF_STATUS_SUCCESS;
    }

    if (!ipf_seek(pf, offset))
        return pf->last_error;

    if (pf->end_of_file || pf->offset == pf->encrypted_part_plain.size) {
        pf->end_of_file = true;
        *bytes_read = 0;
        return PF_STATUS_SUCCESS;
    }

    size_t bytes = ipf_read(pf, output, size);
    if (!bytes)
        return pf->last_error;

    *bytes_read = bytes;
    return PF_STATUS_SUCCESS;
}

pf_write

pf_status_t pf_write(pf_context_t* pf, uint64_t offset, size_t size, const void* input) {
    if (!g_initialized)
        return PF_STATUS_UNINITIALIZED;

    if (!ipf_seek(pf, offset))
        return pf->last_error;

    if (ipf_write(pf, input, size) != size)
        return pf->last_error;
    return PF_STATUS_SUCCESS;
}

pf_flush

pf_status_t pf_flush(pf_context_t* pf) {
    if (!g_initialized)
        return PF_STATUS_UNINITIALIZED;

    if (!ipf_internal_flush(pf))
        return pf->last_error;
    return PF_STATUS_SUCCESS;
}

Code in SDK

Flush

• Flush if status is OK.

bool protected_fs_file::pre_close(sgx_key_128bit_t* key, bool import)
{
    int32_t result32 = 0;
    bool retval = true;
    sgx_status_t status = SGX_SUCCESS;

    sgx_thread_mutex_lock(&mutex);

    if (import == true)
    {
        if (use_user_kdk_key == 1) // import file is only needed for auto-key
            retval = false;
        else
            need_writing = true; // will re-encrypt the neta-data node with local key
    }

    if (file_status != SGX_FILE_STATUS_OK)
    {
        sgx_thread_mutex_unlock(&mutex);
        clear_error(); // last attempt to fix it
        sgx_thread_mutex_lock(&mutex);
    }
    else // file_status == SGX_FILE_STATUS_OK
    {
        internal_flush(/*false,*/ true);
    }

    if (file_status != SGX_FILE_STATUS_OK)
        retval = false;

    if (file != NULL)
    {
        status = u_sgxprotectedfs_fclose(&result32, file);
        if (status != SGX_SUCCESS || result32 != 0)
        {
            last_error = (status != SGX_SUCCESS) ? status : 
                         (result32 != -1) ? result32 : SGX_ERROR_FILE_CLOSE_FAILED;
            retval = false;
        }

        file = NULL;
    }

    if (file_status == SGX_FILE_STATUS_OK && 
        last_error == SGX_SUCCESS) // else...maybe something bad happened and the recovery file will be needed
        erase_recovery_file();

    if (key != NULL)
    {
        if (use_user_kdk_key == 1) // export key is only used for auto-key
        {
            retval = false;
        }
        else
        {
            if (restore_current_meta_data_key(NULL) == true)
                memcpy(key, cur_key, sizeof(sgx_key_128bit_t));
            else
                retval = false;
        }
    }

    file_status = SGX_FILE_STATUS_CLOSED;

    sgx_thread_mutex_unlock(&mutex);

    return retval;
}

Struct definition

SDK Code

typedef struct _file_data_node
{
    /* these are exactly the same as file_mht_node_t above, any change should apply to both (both are saved in the cache as void*) */
    uint8_t type;
    uint64_t data_node_number;
    file_mht_node_t* parent;
    bool need_writing;
    bool new_node;
    union {
        struct {
            uint64_t physical_node_number;
            encrypted_node_t encrypted; // the actual data from the disk
        };
        recovery_node_t recovery_node;
    };
    /* from here the structures are different */
    data_node_t plain; // decrypted data
} file_data_node_t;

This struct is nearly the same as file_mht_node_t, it seems like these are duplicated but working code.

Steps

1. Write recovery file and set flags if needed
2. Update data and mht nodes if needed
3. Update meta data node and update flags
4. Write data to the disk

• Perform the actual data write in protected_fs_file::write_all_changes_to_disk.
• LRU cache is used here for performance.

bool protected_fs_file::internal_flush(/*bool mc,*/ bool flush_to_disk)
{
    if (need_writing == false) // no changes at all
        return true;

    if (encrypted_part_plain.size > MD_USER_DATA_SIZE && root_mht.need_writing == true) // otherwise it's just one write - the meta-data node
    {
        if (_RECOVERY_HOOK_(0) || write_recovery_file() != true)
        {
            file_status = SGX_FILE_STATUS_FLUSH_ERROR;
            return false;
        }

        if (_RECOVERY_HOOK_(1) || set_update_flag(flush_to_disk) != true)
        {
            file_status = SGX_FILE_STATUS_FLUSH_ERROR;
            return false;
        }

        if (_RECOVERY_HOOK_(2) || update_all_data_and_mht_nodes() != true)
        {
            clear_update_flag();
            file_status = SGX_FILE_STATUS_CRYPTO_ERROR; // this is something that shouldn't happen, can't fix this...
            return false;
        }
    }

    if (_RECOVERY_HOOK_(3) || update_meta_data_node() != true)
    {
        clear_update_flag();
        file_status = SGX_FILE_STATUS_CRYPTO_ERROR; // this is something that shouldn't happen, can't fix this...
        return false;
    }

    if (_RECOVERY_HOOK_(4) || write_all_changes_to_disk(flush_to_disk) != true)
    {
        //if (mc == false)
            file_status = SGX_FILE_STATUS_WRITE_TO_DISK_FAILED; // special case, need only to repeat write_all_changes_to_disk in order to repair it
        //else
            //file_status = SGX_FILE_STATUS_WRITE_TO_DISK_FAILED_NEED_MC; // special case, need to repeat write_all_changes_to_disk AND increase the monotonic counter in order to repair it

        return false;
    }

    need_writing = false;

    return true;
}

Struct definition

typedef struct _meta_data_encrypted
{
    char          clean_filename[FILENAME_MAX_LEN];
    int64_t       size;
    
    sgx_mc_uuid_t mc_uuid; // not used
    uint32_t      mc_value; // not used

    sgx_aes_gcm_128bit_key_t mht_key;
    sgx_aes_gcm_128bit_tag_t mht_gmac;

    uint8_t       data[MD_USER_DATA_SIZE];
} meta_data_encrypted_t;